/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package plagiadetector.detector;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import plagiadetector.datatype.GramCounter;
import plagiadetector.datatype.TextSegment;



/**
 * Implementation of Stamatatos' system.
 * Modification of StamaDetector.java, where you can set the default window length.
 * use setDefaultWindowLength(int length) to do this.
 * The window step will be set automatically to 0.2*window length,
 * and the special criterion threshold will be set automatically to 1.5*windowlength.
 * [READ : YOU CAN CHANGE THE PREPROCESS - EXTRACTION STEP TO WRITE TO OUTPUT FILE, SO YOU DON'T HAVE TO RESET ALL THE PROCESS.]
 * @author Dell
 */
public class StamaDetectorBestSettings extends Detector{

    private int ngram; //n-gram size
    private int windowlength; //standard window length - number of minimum letter chars in a window
    private int windowstep; //sliding window'gram step
    private float plagfreethreshold; //plagiarism free criterion threshold for segments.
    private int specialcriterionthreshold; //special criterion threshold.
    private int sensitivity; 
    
    public ArrayList<String> docgrams; //n-grams from the whole text, with all n-gram without letter char removed.
    public ArrayList<ArrayList<String>> windowgrams; //n-grams of windows, sequential.
    
    public HashMap<String, GramCounter> docprofile; //processed document'gram profile - array of n-grams and it'gram occurence.
    public ArrayList<Float> wschange; // style change amount of windows, sequential.
    
    public ArrayList<Float> leftwindowstylechange; //style change container for windows that doesn't met likely-plagiarised heuristic
    public ArrayList<Integer> leftwindowoffset; //preserving position of window location in leftwindowstylechange
    public ArrayList<String> leftwindowtext; //preserving text of left windows
    public ArrayList<Integer> leftwindownumber; //preserving window number of left windows
    private StringBuilder grambuff; //used for building strings on extraction.
  
    @Override
    public void setRawText(String s) {
        super.setRawText(s);

       if(s.length() < 41832){
           setWindowLength(625);
       }else if(s.length() < 346129){
            setWindowLength(1500);
       }else{
           setWindowLength(1750);
       }

            setWindowStep((int)(getWindowLength()*0.2));
            setWindowLengthThreshold((int)(getWindowLength()*1.5f));
    }
    
    
    @Override
    public void doPreprocessing() {
         this.preprocessedText = preprocessText(getRawText());
    }

    @Override
    public String preprocessText(String s) { 
        return s.toLowerCase();
    }

    /**
     * Create sliding windows (segments) from the preprocessed text
     */
    @Override
    public void doSegmentation() {
        StringBuilder strbuff = new StringBuilder();
        int strbufflength; //size of letter text in the window.
        char extracharbuff; //buffer for extra added character.
        int extracharlength; //number of added extra chars. 
        int numofwindows = getNumOfWindows();
        int substringlength;
        int textlength = getPreprocessedText().length();
        
        for(int i=0; i < numofwindows; i++){
            extracharlength = 1; //used to control the extra text
            
            //get the window, move along the step.
            substringlength = (i*windowstep)+windowlength;
            
            if(substringlength+1<=textlength){
                strbuff.append(getPreprocessedText().substring((i*windowstep), substringlength+1)); //increase one, because the code takes until endIndex-1.
            
            }else{ //if the left characters is not enough
                strbuff.append(getPreprocessedText().substring((i*windowstep), textlength));
            }
            
            //make sure that the window contains letters as much as the windowlength.
            strbufflength = strbuff.toString().replaceAll("[^a-z]", "").length();
            
            //if the window not contains enough letter (windowsize), add more letters.
            while((strbufflength < windowlength) && (substringlength+extracharlength < textlength)){
               extracharbuff = getPreprocessedText().charAt(substringlength+extracharlength);
                
                if(Character.isLetter(extracharbuff)){
                    strbufflength++;
                }
                
                strbuff.append(extracharbuff);
                
                extracharlength++;
            }
            
            //add the segment.
            addTextSegment(strbuff.toString(), (i*windowstep));
            
            //clear the strbuff.
            strbuff.replace(0, strbuff.length(), "");
        }
    }

    /**
     * This phase consist of :
     * 1. Extraction of n-grams for each windows
     * 2. Extraction of n-grams from the whole text
     */
    @Override
    public void doStylometricExtraction() {
        
        //extraction of n-grams for each windows
        for(int i = 0; i < getNumOfTextSegments(); i++){
            windowgrams.add(extractNGrams(getTextSegment(i)));
        }
        
        //extraction of n-grams from the whole text.
        //in this process, each n-grams that doesn't have letter chars will be thrown.
        StringBuilder grambuff = new StringBuilder();
        int textlength = getPreprocessedText().length();
        int availabletextlength = textlength - getNGramSize()+1;
        int grambufflength;

        for(int i = 0; i < availabletextlength; i++){
            
            for(int j = 0; j < ngram; j++){
                    grambuff.append(getPreprocessedText().charAt(i+j));
            }

            //check whether the n-gram has any letter character on it.
            if(grambuff.toString().replaceAll("[^a-z]","").length() != 0){//if true, then add.
                docgrams.add(grambuff.toString());
            }
            
//            if(grambuff.toString().equals("vai")) System.out.println("FOUND GRAM vai at CHAR NUMBER "+i+", NEXT CHAR : "+getPreprocessedText().charAt(i+getNGramSize()));
            
            //clear, instead creating a new one.
            grambufflength = grambuff.length();
            grambuff.replace(0, grambufflength, "");
        }
       grambuff = null; //free
    }


    /**
     * Stylometric Quantification :
     * 1. Create document gram profile
     * 2. Count each window gram style change amount
     * 
     * Stylometric Analysis :
     * 1. Apply plagiarism-free criterion heuristic to the document.
     * 2. Apply likely-plagiarised passage heuristic to each window.
     * 3. Check each of left windows with plagiarised passage criterion.
     * 4. Check each windows that fulfills plagiarised passage criterion with special criterion heuristic.
     * 5. Add each windows that not fulfills special criterion to case.
     */
    @Override
    public void doStylometricQA() {
        
        float meanbuff;
        float stdbuff;
        
        int x; 
        boolean found;
        int textlengthbuff;
            
        //Quantification
        buildDocumentProfile();
        buildWindowStyleChange();
        
        //Analysis
        if(!isCurrentDocPlagFree()){
            // plagiarism-free criterion heuristic.
            //if the current document is plagiarism free, then there would be no cases.
        
            //variables used to combine sided cases.
            x = 0; 
            found = false;
            textlengthbuff = 0;
            
            meanbuff = getAllWindowStyleMean();
            stdbuff = getAllWindowStyleStandardDeviaton();
            int wschangesize = wschange.size();
            //[LIKELY-PLAGIARISED PASSAGE HEURISTIC]
            for(int i = 0; i < wschangesize; i++){
                if(wschange.get(i) > (meanbuff+stdbuff)){
                    //if it fits the heuristic
             
                    if(!found){
                        //if this is the first detected segment    
                      
                        x = i;
                        found = true;
                        textlengthbuff = getTextSegment(i).segmentText.length();
                    }else{
                        //if the segment before this was marked too
                      
                        textlengthbuff += getTextSegment(i).offset + getTextSegment(i).segmentText.length() - (getTextSegment(x).offset + textlengthbuff);
                    }
                
                }else{
                    if(found){
                         // if the segment before this was detected as plagiarism case, but this segment isn't
                            addCase(getTextSegment(x).offset,textlengthbuff);
                            found = false;
                    }
                    
                    //yet - whatever happens, add this to left windows
                    leftwindowstylechange.add(wschange.get(i));
                    leftwindowoffset.add(getTextSegment(i).offset);
                    leftwindowtext.add(getTextSegment(i).segmentText);
                    leftwindownumber.add(i);
                }
            }
            
            if(found){
                //if the last window is plagiarism case, and haven't added to the case data
                addCase(getTextSegment(x).offset, textlengthbuff);
            }
                    
            //[PLAGIARISED PASSAGE CRITERION CHECK]
            meanbuff = countWindowStyleMean(leftwindowstylechange);
            stdbuff = countWindowStyleStandardDeviaton(leftwindowstylechange);
            x = 0;
            found = false;
            textlengthbuff = 0;
            
            int leftwindowstylechangesize = leftwindowstylechange.size();
            
            for(int i=0; i < leftwindowstylechangesize; i++){//for all windows that left.
                //check whether the window fits plagiarised passages criterion
                if(leftwindowstylechange.get(i) > (meanbuff + (stdbuff*sensitivity))){
                    
                    //[SPECIAL CRITERION CHECK]
                    if(!(leftwindowtext.get(i).length() < specialcriterionthreshold)){ 
                        //if it doesn't fit the special criterion
                       
                        if(!found){
                            //if this is the first detected segment
                            x = i;
                            found = true;
                            textlengthbuff = leftwindowtext.get(i).length();
                        }else if(leftwindownumber.get(i)-1 == leftwindownumber.get(i-1)){
                            //if the left segment before this was marked,
                            //and it is actually a segment before this one (in preprocessed text)
                            textlengthbuff += leftwindowoffset.get(i) + leftwindowtext.get(i).length() - (leftwindowoffset.get(x) + textlengthbuff);
                        }else{
                            //if the left segment before this was marked,
                            //but not a segment before this one
                            
                            //add the last detected segment to the cases.
                            addCase(leftwindowoffset.get(x), textlengthbuff);
                            
                            //prepare a new one.
                            x = i;
                            textlengthbuff = leftwindowtext.get(i).length();
                        }  
//                        addCase(leftwindowoffset.get(i),leftwindowtext.get(i).length());
                        
                    }else if(found){
                        //if last segment was detected as plagiarism, but this isn't
                        addCase(leftwindowoffset.get(x), textlengthbuff);
                        found = false;
                    }
                }
            }
            
            if(found){
                //if the last window is plagiarism case, and haven't added to the case data
                addCase(getTextSegment(x).offset, textlengthbuff);
            }
        }
        else{
//            System.out.println("This document is free from plagiarism - by heuristic");
        }
    }
    
        /**
     * Constructor of StamaDetector object.
     */
    public StamaDetectorBestSettings(){
        super();
        initialize();
        resetParameters();
    }
    
    /**
     * Constructor of StamaDetector object.
     * @param s Raw text for this object.
     */
    public StamaDetectorBestSettings(String s){
        super();
        setRawText(s);
        initialize();
        resetParameters();
    }
    
    @Override
    public void initialize() {
        super.initialize();
        
        
        //clear all the list.
        
        if(docgrams == null){
            docgrams = new ArrayList<String>();
        }else{
            docgrams.clear();
        }
        
        if(windowgrams == null){
            windowgrams = new ArrayList<ArrayList<String>>();
        }else{
            windowgrams.clear();
        }
        
        if(docprofile == null){
            docprofile = new HashMap<String, GramCounter>();
        }else{
            docprofile.clear();
        }
        
        if(wschange == null){
            wschange = new ArrayList<Float>();
        }else{
            wschange.clear();
        }
        
        if(leftwindownumber == null){
            leftwindownumber = new ArrayList<Integer>();
        }else{
            leftwindownumber.clear();
        }
        
        if(leftwindowoffset == null){
            leftwindowoffset = new ArrayList<Integer>();
        }else{
            leftwindowoffset.clear();
        }
        
        if(leftwindowstylechange == null){
            leftwindowstylechange = new ArrayList<Float>();
        }else{
            leftwindowstylechange.clear();
        }
        
        if(leftwindowtext == null){
            leftwindowtext = new ArrayList<String>();
        }else{
            leftwindowtext.clear();
        }        
                
        if(grambuff == null){
            grambuff = new StringBuilder();
        }else{
            grambuff.replace(0, grambuff.length(), "");
        }
    }
    
    
    /**
     * Set the character n-gram size of this object.
     * @param n preferred n-gram size.
     */
    public void setNGramSize(int n){
        ngram = n;
    }
    
    
    /**
     * Get the character n-gram size of this object.
     * @return this object'gram n-gram size.
     */
    public int getNGramSize(){
        return ngram;
    }
    
    /**
     * @return the windowlength' size
     */
    public int getWindowLength() {
        return windowlength;
    }

    /**
     * @param windowlength the windowlength to set
     */
    public void setWindowLength(int windowlength) {
        this.windowlength = windowlength;
    }

    /**
     * @return the windowstep
     */
    public int getWindowStep() {
        return windowstep;
    }

    /**
     * @param windowstep the windowstep to set
     */
    public void setWindowStep(int windowstep) {
        this.windowstep = windowstep;
    }

    /**
     * @return the plagfreethreshold
     */
    public float getPlagfreeThreshold() {
        return plagfreethreshold;
    }

    /**
     * @param plagfreethreshold the plagfreethreshold to set
     */
    public void setPlagfreeThreshold(float plagfreethreshold) {
        this.plagfreethreshold = plagfreethreshold;
    }

    /**
     * @return the specialcriterionthreshold
     */
    public int getWindowLengthThreshold() {
        return specialcriterionthreshold;
    }

    /**
     * @param specialcriterionthreshold the specialcriterionthreshold to set
     */
    public void setWindowLengthThreshold(int windowlengththreshold) {
        this.specialcriterionthreshold = windowlengththreshold;
    }

    /**
     * @return the sensitivity
     */
    public int getSensitivity() {
        return sensitivity;
    }

    /**
     * @param sensitivity the sensitivity to set
     */
    public void setSensitivity(int sensitivity) {
        this.sensitivity = sensitivity;
    }
    
     /**
     * Gets the number of sliding windows to process the document.
     * @return 
     */
    public int getNumOfWindows(){
        return (int) Math.floor(1+((preprocessedText.length()-windowlength)/windowstep));
    }
    
    /**
     * Extract n-grams from given text segment.
     * @param textseg TextSegment object.
     * @return ArrayList of the n-grams.
     */
    public ArrayList<String> extractNGrams(TextSegment textseg){
        return extractNGrams(textseg.segmentText);
    }
    
    /**
     * Extract n-grams from given text.
     * @param text input text.
     * @return ArrayList of the n-grams
     */
    public ArrayList<String> extractNGrams(String text){
        ArrayList<String> ngramlist = new ArrayList<String>();
       
        int availabletextlength = text.length()-getNGramSize()+1;
        grambuff.replace(0, grambuff.length(), ""); //initiating grambuff.
        
        for(int i = 0; i < availabletextlength; i++){
            for(int j = 0; j < getNGramSize(); j++){
                grambuff.append(text.charAt(i+j));
            }
            
            //check whether the n-gram has any letter character on it.
            if(grambuff.toString().replaceAll("[^a-z]","").length() != 0){//if true, then add.
                ngramlist.add(grambuff.toString());
            }
            
            grambuff.replace(0, grambuff.length(), "");
        }

        return ngramlist;
    }
    
    /**
     * Extract unique n-grams and its frequencies from given ArrayList of String.
     * @param als ArrayList of String object.
     * @return HashMap of GramCounter of unique n-grams
     */
    public HashMap<String,GramCounter> extractNGramFrequency(ArrayList<String> als){
                HashMap<String, GramCounter> agc = new HashMap<String, GramCounter>();

        GramCounter gc;
       
        for(int i = 0; i < als.size(); i++){

            gc = agc.get(als.get(i));
            
            if(gc == null){// new n-gram
                agc.put(als.get(i),new GramCounter(als.get(i), 1f));
            }else{//increment
                ((GramCounter)agc.get(als.get(i))).occurrence++;
            }
        }
        
        return agc;
    }
    
    /**
     * Creates the vector of n-grams of this document, for the document style
     */
    public void buildDocumentProfile(){
       
       //extract n-gram frequency from docgrams
       docprofile = extractNGramFrequency(docgrams);
       
       //normalize the profile with text length
       int textlength = getPreprocessedText().length();
       Iterator i = docprofile.entrySet().iterator();
       String strbuff;
       
       while(i.hasNext()){
           strbuff = (String) ((Map.Entry)i.next()).getKey();
           docprofile.get(strbuff).occurrence /= (float) textlength;         
       }

    }
    
     /**
     * Calculate each sliding window n-gram style change amount
     */
    public void buildWindowStyleChange(){
        
        HashMap<String, GramCounter> windowgc;
        int i = -1; //counter. will be started at 0.
        int ngramcounter;
        float wscbuff; //window style change amount buffer
        float docgramstyle;
        int textlength = getPreprocessedText().length();
        Iterator itx;
        String keybuff;
        float occurencebuff;
        
        for(ArrayList<String> als : windowgrams){ //for each window
            i++;
            
            windowgc = extractNGramFrequency(als); //extract n-gram frequency
            
            ngramcounter = als.size();
            //please comment this print out later.
 /*
            System.out.println("################################################");
             System.out.println("SEGMENT NUMBER "+i+" QUANTIFIED FEATURES, NUMBER OF UNIQUE FEATURES : "+windowgc.size());
             System.out.println("################################################");
            
             itx = windowgc.entrySet().iterator();
             GramCounter gc;
             while(itx.hasNext()){
                gc = (GramCounter)((Map.Entry)itx.next()).getValue();
                System.out.println("Gram : ["+gc.text+"] , Freq : "+gc.occurrence/(float)textlength);
             } 
*/           
            wscbuff = 0f;
            itx = windowgc.entrySet().iterator();
            
            
            //normalize the frequency of the n-grams, then count style change amount of this window
            while(itx.hasNext()){ //for each unique n-gram
                
                keybuff = (String)((Map.Entry)itx.next()).getKey();
           
                windowgc.get(keybuff).occurrence /= (float) textlength;
                
                docgramstyle = 0f;
                
                if(keybuff.replaceAll("[^a-z]", "").length() != 0){
                    //only process n-gram that contains letter alphabet,
                    //because if it's not, the document n-gram frequency must be 0.
                
                    //search the occurence of this n-gram on the document
                    if(docprofile.get(keybuff) != null){
                        docgramstyle = docprofile.get(keybuff).occurrence;
                        
//                    System.out.println("PROCESSING N-GRAM "+keybuff+", WINDOW FREQ : "+windowgc.get(keybuff).occurrence+", DOC FREQ : "+docgramstyle);

                    }else{
//                        System.out.println("BUG FOUND ! KEY "+keybuff+" IS NOT AVAILABLE IN THE DOCUMENT");
                    }               
                    
                }
                
                occurencebuff = windowgc.get(keybuff).occurrence;
                //count this n-gram style change difference.
//                float pow = (float) Math.pow((2*(occurencebuff - docgramstyle) /(double)(occurencebuff+docgramstyle)),2);
                float pow = (2*(occurencebuff - docgramstyle) /(float)(occurencebuff+docgramstyle));
                pow *= pow;
                wscbuff += pow;   
            }
            
            //normalize this window's style change with this window's profile size times four
            wscbuff /= (float) (4*ngramcounter);
            
            //add this window style change to wschange.
            wschange.add(wscbuff);
        }
    }
    
    /**
     * Count mean of style change from given list of window styles.
     * @param windowstyle given list of window styles
     * @return mean amount of the window styles.
     */
    public float countWindowStyleMean(ArrayList<Float> windowstyle){
         float mean = 0f;
         for(Float wsb : windowstyle){
            mean += wsb;
        }
        return (mean/windowstyle.size());
    }
    
    /**
     * Get mean of all sliding windows. The style change window should be initiated first.
     * @return all sliding windows mean value.
     */
    public float getAllWindowStyleMean(){
         return countWindowStyleMean(wschange);
    }
    
        
    /**
     * Count standard deviation of style change from given list of window styles.
     * @param windowstyle given list of window styles
     * @return standard deviations amount of the window styles.
     */
    public float countWindowStyleStandardDeviaton(ArrayList<Float> windowstyle){
         float sum = 0f;
         float buffws;
         float wsmean = countWindowStyleMean(windowstyle);
         for(Float wsb : windowstyle){
            buffws = (wsb-wsmean);
            sum += buffws*buffws;
        }
        return (float) Math.sqrt(sum/(double)windowstyle.size());
    }
    
    /**
     * Get mean of all sliding windows. The style change window should be initiated first.
     * @return all sliding windows mean value.
     */
    public float getAllWindowStyleStandardDeviaton(){
         return countWindowStyleStandardDeviaton(wschange);
    }
    
    /**
     * Check whether the document is plagiarism free. The style change window should be initiated first.
     * @return document's plagiarism free status
     */
    public boolean isCurrentDocPlagFree(){
        
        return (getAllWindowStyleStandardDeviaton() < plagfreethreshold);
    }

    @Override
    public void resetParameters() {
          //set the parameter to default.
        
        if(ngram != 3){
            ngram = 3;
        } //default size
        
        if(windowlength != 1000){
            windowlength = 1000;
        }
        
        if(windowstep != 200){
            windowstep = 200;
        }
        
        if(plagfreethreshold != 0.02f){
            plagfreethreshold = 0.02f;
        }
        
        if(specialcriterionthreshold != 1500){
            specialcriterionthreshold = 1500;
        }
        
        if(sensitivity != 2){
            sensitivity = 2;
        }
    }
}

